Comparison of Sentiment Tools across Domains

# load required libraries

# to use harry potter dataset
# devtools::install_github("bradleyboehmke/harrypotter")
# devtools::install_github("quanteda/quanteda.sentiment")
# devtools::install_github("quanteda/quanteda.corpora")

library(quanteda)
library(readtext)
library(corpus)
library(tidyverse)
library(stringr)
library(tidytext)
library(harrypotter)
library(dplyr)
library(quanteda.sentiment)
library(vader)
library(caret)
library(reshape2)


require(quanteda)
require(quanteda.corpora)
require(quanteda.sentiment)
#library("quanteda", warn.conflicts = FALSE, quietly = TRUE)

1. Step: Load Data & Lexicons

lsd

Dictionary object with 4 key entries.
Polarities: pos = "positive", "neg_negative"; neg = "negative", "neg_positive" 
- [negative]:
  - a lie, abandon*, abas*, abattoir*, abdicat*, aberra*, abhor*, abject*, abnormal*, abolish*, abominab*, abominat*, abrasiv*, absent*, abstrus*, absurd*, abus*, accident*, accost*, accursed* [ ... and 2,838 more ]
- [positive]:
  - ability*, abound*, absolv*, absorbent*, absorption*, abundanc*, abundant*, acced*, accentuat*, accept*, accessib*, acclaim*, acclamation*, accolad*, accommodat*, accomplish*, accord, accordan*, accorded*, accords [ ... and 1,689 more ]
- [neg_positive]:
  - best not, better not, no damag*, no no, not ability*, not able, not abound*, not absolv*, not absorbent*, not absorption*, not abundanc*, not abundant*, not acced*, not accentuat*, not accept*, not accessib*, not acclaim*, not acclamation*, not accolad*, not accommodat* [ ... and 1,701 more ]
- [neg_negative]:
  - not a lie, not abandon*, not abas*, not abattoir*, not abdicat*, not aberra*, not abhor*, not abject*, not abnormal*, not abolish*, not abominab*, not abominat*, not abrasiv*, not absent*, not abstrus*, not absurd*, not abus*, not accident*, not accost*, not accursed* [ ... and 2,840 more ]

reviews

twitter

parlvote

2. Step: Perform Sentiment Analysis

Normalize Scores

# DATA NORMALIZATION
# Normalize data via minimun/maximum normalization, either by scaling values from 0 to 1 or -1 to 1
# 
# Arg:
#   x: input values (e.g. column of data frame)
#   
# Returns: 
#   normalized data


# min/max normalization from 0 to 1
normalize <- function(x, na.rm = TRUE){
  return((x-min(x)) / (max(x)-min(x)))}

# min/max normalization from -1 to 1
normalize2 <- function(x, na.rm = TRUE){
  return(2* ((x - min(x)) / (max(x)-min(x)))-1)}

Calculate Sentiment Scores

3. Step: Calculate Statistics

Convert data into ternary format

# Convert final values into ternary (1 = positive, 0 = neutral, -1 = negative) format for evaluation and comparison
# 
# Arg:
#   df: input data frame that contains the columns to be converted into ternary format
#   to_change: column names that should be converted into ternary format
#   gold: goldstandard, i.e. rating, which has to be changed to ternary format
#   
# Returns: 
#   data frame with converted values

get_binary <- function(df, to_change, gold){
  df %>% 
    mutate_at(to_change, function(x){
      # mutate values greater than 0 to 1 (positive), equal to 0 to 0 (neutral) and smaller than 0 to -1 (negative)
      case_when(x > 0 ~ 1, x < 0 ~ -1, x == 0 ~ 0)}) %>% 
    
    mutate_at(gold, function(x){
      # if x is between 0-5, mutate values greater than 0 to 1, equal to 0 to 0 and smaller than 0 to -1
      case_when(between(x,0,5) & x > 3 ~ 1, between(x,0,5) & x == 3 ~ 0, between(x,0,5) & x < 3 ~ -1,
        x == "Positive" ~ 1, x == "Negative" ~ -1, x == "Neutral" ~ 0)})
}

reviews_binary <- get_binary(reviews_sentiment, c("afinn","vader","lsd"), "rating")
twitter_binary <- get_binary(twitter_sentiment, c("afinn","vader","lsd"), "rating")

Warnung: Problem with `mutate()` column `rating`.
ℹ `rating = (function (x) ...`.
ℹ NAs durch Umwandlung erzeugt
Warnung: Problem with `mutate()` column `rating`.
ℹ `rating = (function (x) ...`.
ℹ NAs durch Umwandlung erzeugt
Warnung: Problem with `mutate()` column `rating`.
ℹ `rating = (function (x) ...`.
ℹ NAs durch Umwandlung erzeugt

parlvote_binary <- get_binary(parlvote_sentiment, c("afinn","vader","lsd"), "rating")

Calculate Accuracy, Precision, Recall

# Function to get statistics (accuracy, precision, recall) of data frame for specific lexicon
#
# Arg: 
#   df: data frame that we want statistics of
#   lexicon: binary/ternary lexicon that should be evaluated
#
# Returns:
#   overall accuracy score
#   data frame with statistics
  
get_statistics <- function(df, lexicon){
  
  # create confusion matrix (via caret library) to get statistics of data frame
  cm <- confusionMatrix(factor(df[[lexicon]]), factor(df$rating), mode="prec_recall")
  
  acc <- (as.data.frame(cm$overall))["Accuracy",]
  stats <- as.data.frame(cm$byClass)
  results <- list(acc, stats)
  
  # check how to return two statements
  return(results)
}

reviews_afinn.acc <- get_statistics(reviews_binary, "afinn")[[1]][1]
reviews_afinn.stats <- get_statistics(reviews_binary, "afinn")[2]

reviews_lsd.acc <- get_statistics(reviews_binary, "lsd")[[1]][1]
reviews_lsd.stats <- get_statistics(reviews_binary, "lsd")[2]

reviews_vader.acc <- get_statistics(reviews_binary, "vader")[[1]][1]
reviews_vader.stats <- get_statistics(reviews_binary, "vader")[2]

4. Step: Plot Data

# plot columns
reviews_dfm <- melt(head(reviews_sentiment,50)[,c('id','afinn','lsd','vader')],id.vars = 1)

reviews_plot <- ggplot(reviews_dfm,aes(x = id,y = value)) + 
                geom_bar(aes(fill = variable),stat = "identity",position = "dodge") +
                ggtitle("Reviews Sentiments")


twitter_dfm <- melt(head(twitter_sentiment,50)[,c('id','afinn','lsd','vader')],id.vars = 1)

twitter_plot <- ggplot(twitter_dfm,aes(x = id,y = value)) + 
                geom_bar(aes(fill = variable),stat = "identity",position = "dodge") +
                ggtitle("Twitter Sentiments")

parlvote_dfm <- melt(head(parlvote_sentiment,50)[,c('id','afinn','lsd','vader')],id.vars = 1)

parlvote_plot <- ggplot(parlvote_dfm,aes(x = id,y = value)) + 
                geom_bar(aes(fill = variable),stat = "identity",position = "dodge")+
                facet_wrap(~ variable, ncol = 1, scales="free_y")+
                ggtitle("ParlVote Sentiments")

#parlvote_norm_dfm <- melt(head(parlvote_sentiment,50)[,c('id','afinn_norm','lsd_norm','vader_norm')],id.vars = 1)

#parlvote_norm_plot <- ggplot(parlvote_norm_dfm,aes(x = id,y = value)) + 
              #  geom_bar(aes(fill = variable),stat = "identity",position = "dodge")+
               # facet_wrap(~ variable, ncol = 1, scales="free_y")+
                #ggtitle("ParlVote Sentiments")


reviews_plot

twitter_plot

parlvote_plot

#parlvote_norm_plot

Plot important words

# optional

Ranking of texts per lexicon

# sort per corpus and per tool
reviews_afinn.sort <- reviews_sentiment[order(reviews_sentiment$afinn, decreasing=TRUE),]

reviews_lsd.sort <- reviews_sentiment[order(reviews_sentiment$lsd, decreasing=TRUE),]

reviews_vader.sort <- reviews_sentiment[order(reviews_sentiment$vader, decreasing=TRUE),]


reviews_afinn.sort

reviews_lsd.sort

reviews_vader.sort

5. Step: Evaluation (Notes)

• evaluate data/ compare data
• accuracy, precision, recall
• pearsons coefficient

Comparison Groups: - compare binary (normalized) versions - each lexicon and data set - compare continuous (normalized) versions - each lexicon and data set - compare top n words per sentiment per tool? (= contribution to sentiment) - compare ranking (Stede’s idea): - rank texts per corpus and compare across tools -> is order similar? - tool’s performance - how is performance across domains? - is binary more accurate than continuous scoring? -

ADDITIONAL

Harry Potter - Dataset

# load harry potter dataset 
titles <- c("Philosopher's Stone", "Chamber of Secrets", "Prisoner of Azkaban",
            "Goblet of Fire", "Order of the Phoenix", "Half-Blood Prince",
            "Deathly Hallows")

books <- list(philosophers_stone, chamber_of_secrets, prisoner_of_azkaban,
           goblet_of_fire, order_of_the_phoenix, half_blood_prince,
           deathly_hallows)
  
series <- tibble()

for(i in seq_along(titles)) {
        
        clean <- tibble(chapter = seq_along(books[[i]]),
                        text = books[[i]]) %>%
             unnest_tokens(word, text) %>%
             mutate(book = titles[i]) %>%
             select(book, everything())

        series <- rbind(series, clean)
}

series$book <- factor(series$book, levels = rev(titles))

series

#book_groups <- series %>% group_by(book, chapter)
# tokenize hp1
#hp1_tokenized <- tokens_tolower(tokens(philosophers_stone, remove_punct = TRUE)) 

afinn_hp <- series %>%
       # group_by(book, chapter) %>% # add word for single word scores 
        #inner_join(get_sentiments("afinn")) %>%
        inner_join(textstat_valence(series$tokens,afinn)$sentiment) %>%
      #  group_by(book, chapter) %>% # add word for single word scores
        #summarise(sentiment = sum(value)) %>%
       # summarise(sentiment = mean(value, na.rm = TRUE)) %>%
        mutate(method = "AFINN")  #%>%

Warnung: Unknown or uninitialised column: `tokens`.
Fehler: textstat_valence() only works on character, corpus, dfm, tokens objects.

Harry Potter - AFINN Lexicon

afinn_hp2 <- series %>%
        group_by(book, chapter) %>% # add word for single word scores 
        inner_join(get_sentiments("afinn")) %>%
        group_by(book, chapter) %>% # add word for single word scores
        #summarise(sentiment = sum(value)) %>%
        summarise(sentiment = mean(value, na.rm = TRUE)) %>%
        mutate(method = "AFINN")  %>%
        ggplot(aes(chapter, sentiment, fill = book)) +
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE) +
          facet_wrap(~ book, ncol = 2, scales = "free_x")

afinn_hp2

#ggsave(plot = afinn, width = 15, height = 15, dpi = 300, filename = "afinn_hp_mean.png")

Lexicoder: HP

# select only the "negative" and "positive" categories
#data_dictionary_LSD2015_pos_neg <- data_dictionary_LSD2015[1:2]
#hp1_lsd <- tokens_lookup(hp1_tokenized, dictionary = data_dictionary_LSD2015_pos_neg)

polarity(data_dictionary_LSD2015) <- 
  list(pos = c("positive", "neg_negative"), neg = c("negative", "neg_positive"))

hp1_lsd <- textstat_polarity(hp1_tokenized, data_dictionary_LSD2015)

hp1_lsd_tokens <- tokens_lookup(hp1_tokenized, data_dictionary_LSD2015, nested_scope = "dictionary", exclusive = FALSE)
hp1_lsd.df <- as.data.frame.matrix(hp1_lsd)
hp1_lsd.df$chapter <- 1:nrow(hp1_lsd.df)

plot <- ggplot(hp1_lsd, aes(x =hp1_lsd.df$chapter, y=sentiment)) +
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE)
plot + ylim(-1.0, 1.0) + labs(y="sentiment", x = "chapter") + ggtitle("HP1 - Lexicoder")

AFINN: HP

hp1_afinn <- textstat_valence(hp1_tokenized, afinn, normalize="dictionary")

hp1_afinn.df <- as.data.frame.matrix(hp1_afinn)
hp1_afinn.df$chapter <- 1:nrow(hp1_afinn.df)

plot <- ggplot(hp1_afinn.df, aes(x =hp1_afinn.df$chapter, y=sentiment)) +
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE)
plot + ylim(-1.0, 1.0) + labs(y="sentiment", x = "chapter") + ggtitle("HP1 - AFINN")

VADER: HP

get_vader(philosophers_stone[1])

hp1_vader <- vader_df(philosophers_stone)
hp1_vader$chapter <- 1:nrow(hp1_vader)

plot <- ggplot(hp1_vader, aes(x =chapter, y=compound)) +
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE)
plot + ylim(-5.0, 5.0) + labs(y="sentiment", x = "chapter") + ggtitle("HP1 - VADER")

QUANTEDA.SENTIMENT

AFINN: HP

# Work with quanteda.sentiment on HP corpus:
# convert tibble to dataframe
series.df <- as.data.frame(series)

# tokenize books
series_tokenized <- series.df %>%
  unnest_tokens(tokens, text)

# apply afinn lexicon
series_tokenized$afinn <- textstat_valence(series_tokenized$tokens, afinn)$sentiment

# replace all 0 values with na
series_tokenized[series_tokenized == 0] <- NA

series_tokenized %>%
  group_by(book, chapter) %>% # group df by book and chapter to get sentiment per chapter
  summarise(sentiment = mean(afinn, na.rm = TRUE)) %>% # calculate mean w/o regarding na values
  mutate(method = "AFINN") %>% # add column with method 
        ggplot(aes(chapter, sentiment, fill = book)) + # plot sentiment of books
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE) +
          facet_wrap(~ book, ncol = 2, scales = "free_x") +
          ggtitle("AFINN HP")

Lexicoder: HP

# Work with quanteda.sentiment on HP corpus:
# apply lexicoder lexicon
series$lsd <- textstat_polarity(tokens(series$text), data_dictionary_LSD2015)$sentiment 

#series.df <- as.data.frame(series)

plot <- ggplot(series, aes(chapter, lsd, fill = book)) + # plot sentiment of books
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE) +
          facet_wrap(~ book, ncol = 2, scales = "free_x") +
          ggtitle("Lexicoder HP")
plot 

Vader: HP

# apply vader lexicon to all HP books
series$vader <- vader_df(series$text)$compound

#series.df <- as.data.frame(series)

plot <- ggplot(series, aes(chapter, lsd, fill = book)) + # plot sentiment of books
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE) +
          facet_wrap(~ book, ncol = 2, scales = "free_x") +
          ggtitle("VADER HP")
plot 

REVIEWS DATASET

# load dataset
reviews <- readtext("datasets/goodreads_reviews_children_2.json", text_field = "review_text")

# convert to dataframe
reviews.df <- as.data.frame(reviews)

# add doc_id (i.e. according to index)
reviews.df$doc_id <- 1:nrow(reviews.df)

Sample Dataset

# get random sample of 50 reviews 
reviews_sample <- reviews.df[sample(1:nrow(reviews.df), 50,
   replace=FALSE),]

# get first 50 rows of data 
reviews_50 <- head(reviews.df,50)
reviews_50 = subset(reviews_50, select = c(doc_id,text,rating))

Get Translations of Dataset

# either via corpus 
reviews.corpus <- corpus(reviews)
docvars(reviews.corpus, "language") <- textcat(reviews.corpus)
reviews_en <- corpus_subset(reviews.corpus, language == "english", drop_docid = TRUE)

# or via dataframe logic
reviews.df$language <- textcat(reviews.df$text)

Sentiment Analysis on Reviews Dataset

AFINN

# Afinn
# tokenize 
reviews_tokenized <- reviews_50 %>%
  unnest_tokens(tokens, text)

# apply afinn lexicon
reviews_tokenized$afinn <- textstat_valence(reviews_tokenized$tokens, afinn)$sentiment

# replace all 0 values with na
reviews_tokenized[reviews_tokenized == 0] <- NA

# calculate mean scores for tokens per doc
afinn_scores <- reviews_tokenized %>%
  group_by(doc_id) %>% # group df by doc_id to get mean sentiment score
  summarise(total = mean(afinn, na.rm = TRUE)) #%>% # calculate mean w/o regarding na values

# add afinn scores to df 
reviews_50$afinn <- afinn_scores$total

# different version to get plot 
reviews_tokenized %>%
  group_by(doc_id) %>% # group df by book and chapter to get sentiment per chapter
  #reviews_tokenized$sentiment = mean(afinn, na.rm = TRUE) %>%
  summarise(sentiment = mean(afinn, na.rm = TRUE)) %>% # calculate mean w/o regarding na values
  mutate(method = "AFINN") %>% # add column with method 
        ggplot(aes(doc_id, sentiment, fill = doc_id)) + # plot sentiment of books
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE) +
          #facet_wrap(~ doc_id, ncol = 2, scales = "free_x") +
          ggtitle("AFINN Reviews")

LEXICODER

# apply lexicoder lexicon
reviews_50$lsd <- textstat_polarity(tokens(reviews_50$text), data_dictionary_LSD2015)$sentiment 
#series.df <- as.data.frame(series)

plot <- ggplot(reviews_50, aes(doc_id, lsd, fill = doc_id)) + # plot sentiment of books
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE) +
          #facet_wrap(~ doc_id, ncol = 2, scales = "free_x") +
          ggtitle("Lexicoder Reviews")
plot 

Vader

reviews_50$vader <- vader_df(reviews_50$text)$compound

plot <- ggplot(reviews_50, aes(doc_id, vader, fill = doc_id)) + # plot sentiment of books
          geom_bar(alpha = 0.8, stat = "identity", show.legend = FALSE) +
          #facet_wrap(~ doc_id, ncol = 2, scales = "free_x") +
          ggtitle("Vader Reviews")
plot 

get_statistics <- function(df) {
  statistics <- data.frame(matrix(ncol=4, nrow=0))
  x <- c("accuracy", "precision", "recall", "F1")
  colnames(statistics) <- x
  lex1 <- "afinn_binary"
  lex2 <- "lsd_binary"
  lex3 <- "vader_binary"
  gold <- "rating_binary"
  
  lexicons <- c(lex1,lex2,lex3)
  
  for(lex in lexicons){
    confusion_matrix <- table(ACTUAL=df[[gold]], PREDICTED=df[[lex]])
    TN <- confusion_matrix[1]
    FN <- confusion_matrix[2]
    FP <- confusion_matrix[3]
    TP <- confusion_matrix[4]
  
    # calculate statistics
    precision <- TP/(TP+FP)
    accuracy <- (TP+TN)/(TP+TN+FP+FN)
    recall <- TP/(TP+FN)
    F1 <- (2*precision*recall)/(precision+recall)
  
    # add to table
    output <- c(accuracy,precision, recall, F1)
    statistics[lex,] = rbind(statistics[[lex]], output)
    }
    
  return(statistics)
  
}

get_statistics(test)

Test Functions

# complete function to get lexicon scores
get_sentiment <- function(data, lexicons){
  
  for(lex in lexicons){
    if(lex == "afinn"){
      
    data$afinn <- textstat_valence(tokens(data$text), afinn, normalize="dictionary")$sentiment
    }
    
    if(lex == "lsd"){
    data$lsd<- textstat_polarity(tokens(data$text), data_dictionary_LSD2015)$sentiment
    }
    
    if(lex == "vader"){
    data$vader <- vader_df(data$text)$compound
    }
  }
  data[is.na(data)] <- 0
  return(data)
}

reviews_sentiment <- get_sentiment(reviews, c("afinn","lsd", "vader"))
twitter_sentiment <- get_sentiment(twitter, c("afinn","lsd", "vader"))
parlvote_sentiment <- get_sentiment(parlvote, c("afinn","lsd", "vader"))

parlvote_sentiment
#write.csv(parlvote_sentiment, "datasets/parlvote_sentiment.csv", row.names = FALSE)

```